package zsd.reiview.experiment.crf;

import java.awt.Dimension;
import java.io.File;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import zsd.reiview.experiment.crf.CrfCorpus.COAE2009TargetCorpus;
import zsd.reiview.experiment.crf.CrfCorpus.Task3Corpus;
import zsd.reiview.experiment.crf.CrfFeatureExtractor.CrfChainCrfFeatures;
import zsd.reiview.utility.StringUtility;
import zsd.review.split.IctSpliting;

import com.aliasi.chunk.Chunking;
import com.aliasi.corpus.ListCorpus;
import com.aliasi.corpus.XValidatingObjectCorpus;
import com.aliasi.crf.ChainCrfFeatureExtractor;
import com.aliasi.crf.ChainCrfFeatures;
import com.aliasi.matrix.CosineDistance;
import com.aliasi.matrix.DenseVector;
import com.aliasi.matrix.SparseFloatVector;
import com.aliasi.symbol.MapSymbolTable;
import com.aliasi.symbol.SymbolTable;
import com.aliasi.tokenizer.CharacterTokenizerFactory;
import com.aliasi.tokenizer.TokenNGramTokenizerFactory;
import com.aliasi.tokenizer.Tokenizer;
import com.aliasi.tokenizer.TokenizerFactory;
import com.aliasi.util.AbstractExternalizable;
import com.aliasi.util.Arrays;
import com.aliasi.util.Counter;
import com.aliasi.util.ObjectToDoubleMap;

import edu.stanford.nlp.io.EncodingPrintWriter.out;

/**
 * @Title: TrainFilter.java
 * @Package zsd.reiview.experiment.crf
 * @Description: TODO(添加描述)
 * @author 朱圣代
 * @date 2011-11-3 下午02:37:15
 * @version V1.0
 */
public class TrainCorpusFilter implements Serializable {

	/**
	 * @Fields serialVersionUID : TODO(用一句话描述这个变量表示什么)
	 */

	String name;

	private static final long serialVersionUID = 7783248388085792401L;
	/**
	 * @Title: main
	 * @Description:
	 * @param @param args
	 * @return void
	 * @throws
	 * @date 2011-11-3 下午02:37:15
	 */
	TokenizerFactory tokenizerFactory = CharacterTokenizerFactory.INSTANCE;
	MapSymbolTable table;
	/*
	 * public HashMap<String, List<String>> featureMap = new HashMap<String,
	 * List<String>>(1024); public HashMap<String, Double> featSimHashMap = new
	 * HashMap<String, Double>(1024); public HashMap<String, List<String>>
	 * tokenMap = new HashMap<String, List<String>>(1024); public HashMap<String,
	 * List<String>> featMap = new HashMap<String, List<String>>(1024); public
	 * HashMap<String, Double> tokenSimHashMap = new HashMap<String,
	 * Double>(1024);
	 */

	public float[][] tranTokMatrix;
	public float[][] simTokMatrix;
	public float[][] tranFeatMatrix;
	public float[][] simFeatMatrix;
	public float[][] tran;
	public float[] initValue;
	public float[] simInitValue;
	public float[] featInitValue;
	public int dim;
	public int trainNum;
	public int testNum;
	public List<String> trainList;
	public List<String> testList;

	public TrainCorpusFilter(String string) {
	}

	public TrainCorpusFilter() {
	}

	public XValidatingObjectCorpus<Chunking> getFilterCorpus(int N, XValidatingObjectCorpus<Chunking> train, XValidatingObjectCorpus<Chunking> test) {
		HashMap<String, Chunking> trainHashMap = new HashMap<String, Chunking>();
		List<Chunking> trainChunkings = train.getAllItem();
		List<String> tainList = new ArrayList<String>();
		for (Chunking chunking : trainChunkings) {
			String str = chunking.charSequence().toString();
			trainHashMap.put(str, chunking);
			tainList.add(str);
		}

		List<Chunking> testChunkings = test.getAllItem();
		List<String> testList = new ArrayList<String>();
		for (Chunking chunking : testChunkings) {
			String str = chunking.charSequence().toString();
			testList.add(str);
		}

		List<String> sortedTainList = simFilting(tainList, testList);

		XValidatingObjectCorpus<Chunking> corpus = new XValidatingObjectCorpus<Chunking>(train.numFolds());
		for (int i = 0; i < N; i++)
			corpus.handle(trainHashMap.get(sortedTainList.get(i)));
		return corpus;

	}

	public List<String> simFilting(List<String> train, List<String> test) {
		List<Instance> instances = new ArrayList<Instance>();
		for (String str : train) {
			Instance instance = new Instance();
			instance.str = str;
			instances.add(instance);
		}

		List<Instance> testInstances = new ArrayList<Instance>();
		for (String str : test) {
			Instance instance = new Instance();
			instance.str = str;
			testInstances.add(instance);
		}

		table = new MapSymbolTable();
		addList(train, table);
		addList(test, table);

		for (Instance instance : testInstances) {
			SparseFloatVector vector = getFloatVector(instance.getStr(), table);
			instance.setVector(vector);
		}

		for (Instance instance : instances) {
			SparseFloatVector vector = getFloatVector(instance.getStr(), table);
			instance.setVector(vector);
			double sim = 0;
			for (Instance testInstance : testInstances) {
				sim += vector.cosine(testInstance.getVector());
			}
			instance.setSim(sim);
		}

		Collections.sort(instances, new InstanceCompair());

		for (int i = 0; i < train.size(); i++)
			train.set(i, instances.get(i).getStr());
		return train;

	}

	public void addList(List<String> list, SymbolTable table) {
		for (String string : list) {
			Tokenizer tokenizer = tokenizerFactory.tokenizer(string.toCharArray(), 0, string.length());
			String next;
			while ((next = tokenizer.nextToken()) != null) {
				table.getOrAddSymbol(next);
			}
		}
	}

	public SparseFloatVector getFloatVector(String str, SymbolTable table) {
		ObjectToDoubleMap<Integer> map = new ObjectToDoubleMap<Integer>();
		Tokenizer tokenizer = tokenizerFactory.tokenizer(str.toCharArray(), 0, str.length());
		String next;
		double sum = 0;
		while ((next = tokenizer.nextToken()) != null) {
			sum += 1.0;
			Integer id = table.getOrAddSymbol(next);
			Double num = map.get(table.getOrAddSymbol(next));
			if (num == null) {
				map.set(id, 1.0);
			} else {
				map.increment(id, 1.0);
			}
		}
		for (Integer keyInteger : map.keySet()) {
			map.set(keyInteger, map.getValue(keyInteger) / sum);
		}
		SparseFloatVector vector = new SparseFloatVector(map, table.numSymbols());
		/*
		 * int[] notZeroDim = vector.nonZeroDimensions(); double sum = 0; for (int i
		 * = 0; i < notZeroDim.length; i++) sum += vector.value(notZeroDim[i]); for
		 * (int i = 0; i < notZeroDim.length; i++) vector.setValue(notZeroDim[i],
		 * vector.value(notZeroDim[i]) / sum);
		 */
		return vector;
	}

	public SparseFloatVector getFloatVector(List<String> str, SymbolTable table) {
		ObjectToDoubleMap<Integer> map = new ObjectToDoubleMap<Integer>();

		double sum = 0;
		for (int i = 0; i < str.size(); i++) {
			String s = str.get(i);
			sum += 1.0;
			Integer id = table.getOrAddSymbol(s);
			Double num = map.get(table.getOrAddSymbol(s));
			if (num == null) {
				map.set(id, 1.0);
			} else {
				map.increment(id, 1.0);
			}
		}
		for (Integer keyInteger : map.keySet()) {
			map.set(keyInteger, map.getValue(keyInteger) / sum);
		}
		SparseFloatVector vector = new SparseFloatVector(map, table.numSymbols());
		/*
		 * int[] notZeroDim = vector.nonZeroDimensions(); double sum = 0; for (int i
		 * = 0; i < notZeroDim.length; i++) sum += vector.value(notZeroDim[i]); for
		 * (int i = 0; i < notZeroDim.length; i++) vector.setValue(notZeroDim[i],
		 * vector.value(notZeroDim[i]) / sum);
		 */
		return vector;
	}

	public static class Instance {
		String str;
		SparseFloatVector vector;
		SparseFloatVector featVector;
		double sim;
		Chunking chunking;

		public SparseFloatVector getVector() {
			return vector;
		}

		public void setVector(SparseFloatVector vector) {
			this.vector = vector;
		}

		public double getSim() {
			return sim;
		}

		public void setSim(double sim) {
			this.sim = sim;
		}

		public String getStr() {
			return str;
		}

		public void setStr(String str) {
			this.str = str;
		}

		public SparseFloatVector getFeatVector() {
			return featVector;
		}

		public void setFeatVector(SparseFloatVector featVector) {
			this.featVector = featVector;
		}

		public Chunking getChunking() {
			return chunking;
		}

		public void setChunking(Chunking chunking) {
			this.chunking = chunking;
		}
	}

	public static class InstanceCompair implements Comparator<Instance> {

		@Override
		public int compare(Instance o1, Instance o2) {
			if (o1.getSim() < o2.getSim())
				return 1;
			if (o1.getSim() > o2.getSim())
				return -1;
			return 0;
		}

	}

	public static class InstanceCompair2 implements Comparator<Instance> {

		@Override
		public int compare(Instance o1, Instance o2) {
			if (o1.getSim() > o2.getSim())
				return 1;
			if (o1.getSim() < o2.getSim())
				return -1;
			return 0;
		}

	}

	int type;

	private int time;

	private boolean update;

	private int split;

	public static class SentenceSimilarity {
		String sent1;
		String sent2;
		double alph;
		TokenizerFactory featureExtractTokenizerFactory;
		TokenizerFactory tokenTokenizerFactory;

		public SentenceSimilarity(String s1, String s2, double a) {
			sent1 = s1;
			sent2 = s2;
			alph = a;
		}

		public double tokenSimilarity() {

			return alph;

		}

		public static double listCommonNum(List<String> toks1, List<String> toks2) {
			HashMap<String, Double> Hash = new HashMap<String, Double>();
			for (String t2 : toks2)
				Hash.put(t2, 1.0);
			double num = 0;
			for (String t1 : toks1)
				if (Hash.containsKey(t1))
					num++;
			return num;
		}

		public static double listCosineSimilarity(List<String> toks1, List<String> toks2) {
			double incre1 = 1 / (double) toks1.size();
			double incre2 = 1 / (double) toks2.size();
			SymbolTable table = new MapSymbolTable();
			for (String str : toks1) {
				table.getOrAddSymbol(str);
			}
			for (String str : toks2) {
				table.getOrAddSymbol(str);
			}
			ObjectToDoubleMap<Integer> map1 = new ObjectToDoubleMap<Integer>();
			ObjectToDoubleMap<Integer> map2 = new ObjectToDoubleMap<Integer>();
			for (String str : toks1) {
				Integer key = table.getOrAddSymbol(str);
				Double double1 = map1.getValue(key);
				if (double1 == null)
					map1.put(key, incre1);
				else {
					map1.increment(key, double1 + incre1);
				}
			}

			for (String str : toks2) {
				Integer key = table.getOrAddSymbol(str);
				Double double1 = map1.getValue(key);
				if (double1 == null)
					map2.put(key, incre2);
				else {
					map2.increment(key, double1 + incre2);
				}
			}
			int dim = table.numSymbols();
			SparseFloatVector vector1 = new SparseFloatVector(map1, dim);
			SparseFloatVector vector2 = new SparseFloatVector(map2, dim);
			CosineDistance cosineDistance = new CosineDistance();
			// double sim = vector1.cosine(vector2);
			double cos = cosineDistance.proximity(vector1, vector2);
			return cos;
		}
	}

	private List<Chunking> trainChunkings;

	public ListCorpus<Chunking> FilterCorpus(String name, int type, int N, double alph, double beita, double gama, ListCorpus<Chunking> train, ListCorpus<Chunking> test, int time, boolean update,
			int split) throws Exception {

		this.name = name;
		this.time = time;
		this.update = update;
		this.split = split;

		testList = new ArrayList<String>();
		trainList = new ArrayList<String>();

		List<Instance> instances = new ArrayList<Instance>();

		trainChunkings = train.trainCases();
		for (Chunking chunking : trainChunkings) {
			String str = chunking.charSequence().toString();
			Instance instance = new Instance();
			instance.setChunking(chunking);
			instances.add(instance);
			trainList.add(str);
		}

		List<Chunking> testChunkings = test.testCases();

		for (Chunking chunking : testChunkings) {
			String str = chunking.charSequence().toString();
			testList.add(str);
		}
		this.trainNum = trainList.size();
		dim = trainNum;
		this.testNum = testList.size();

		simFiltingByTable(type, alph, beita, gama);

		for (int i = 0; i < dim; i++) {
			instances.get(i).setSim(initValue[i]);
		}

		Collections.sort(instances, new InstanceCompair());

		ListCorpus<Chunking> corpus = new ListCorpus<Chunking>();

		for (int i = 0; i < dim && i < N; i++)
			corpus.addTrain(instances.get(i).getChunking());

		return corpus;
	}

	public XValidatingObjectCorpus<Chunking> FilterCorpus(String name, int type, int N, double alph, double beita, double gama, XValidatingObjectCorpus<Chunking> train,
			XValidatingObjectCorpus<Chunking> test, int time, boolean update, int split) throws Exception {

		this.name = name;
		this.time = time;
		this.update = update;
		this.split = split;

		testList = new ArrayList<String>();
		trainList = new ArrayList<String>();

		List<Instance> instances = new ArrayList<Instance>();

		trainChunkings = train.getAllItem();
		for (Chunking chunking : trainChunkings) {
			String str = chunking.charSequence().toString();
			Instance instance = new Instance();
			instance.setChunking(chunking);
			instances.add(instance);
			trainList.add(str);
		}

		List<Chunking> testChunkings = test.getAllItem();

		for (Chunking chunking : testChunkings) {
			String str = chunking.charSequence().toString();
			testList.add(str);
		}
		this.trainNum = trainList.size();
		dim = trainNum;
		this.testNum = testList.size();
		// List<String> sortedTainList = simFilting(alph, beita, gama, tainList,
		// testList);

		simFiltingByTable(type, alph, beita, gama);

		// double[] ranked = GraphRank(beita, gama);

		for (int i = 0; i < dim; i++) {
			instances.get(i).setSim(initValue[i]);
		}

		Collections.sort(instances, new InstanceCompair());

		XValidatingObjectCorpus<Chunking> corpus = new XValidatingObjectCorpus<Chunking>(train.numFolds());

		for (int i = 0; i < dim && i < N; i++)
			corpus.handle(instances.get(i).getChunking());

		return corpus;

	}

	private double[] GraphRank(double beita, double gama) {
		GraphRank graphRank = new GraphRank(dim, beita, gama, initValue, tran);
		double[] ranked = graphRank.rank();
		return ranked;
	}

	public List<String> simFilting(double alph, double beita, double gama, List<String> train, List<String> test) throws Exception {
		List<Instance> instances = new ArrayList<Instance>();
		tokenizerFactory = CharacterTokenizerFactory.INSTANCE;
		List<Instance> testInstances = new ArrayList<Instance>();
		for (String str : test) {
			Instance instance = new Instance();
			instance.str = str;
			testInstances.add(instance);
		}

		int dim = train.size();
		double[] initValue = new double[dim];
		java.util.Arrays.fill(initValue, 0);
		for (int i = 0; i < dim; i++) {
			// ########
			String s1 = train.get(i);
			System.out.println(i + "   " + s1);
			for (int t = 0; t < test.size(); t++) {

				String s2 = test.get(t);
				// System.out.println(i + "  " + t+"   "+ s2);
				double toksim = tokenSimilarity(s1, s2);
				double featsim = featureSimilarity(s1, s2);
				featsim /= 1000;
				double sim = alph * toksim + (1 - alph) * featsim;
				initValue[i] += sim;
			}
			if (i % 500 == 0)
				AbstractExternalizable.serializeTo(this, new File(getPath(name)));
		}

		double[][] tran = new double[dim][dim];
		for (int i = 0; i < dim; i++) {
			String s1 = train.get(i);
			System.out.println(i + "   " + s1);
			for (int j = 0; j < i; j++) {
				String s2 = train.get(j);
				double toksim = tokenSimilarity(s1, s2);
				double featsim = featureSimilarity(s1, s2);
				double sim = alph * toksim + (1 - alph) * featsim;
				tran[i][j] = sim;
				tran[j][i] = sim;
			}
			tran[i][i] = 0;
			if (i % 2000 == 0)
				AbstractExternalizable.serializeTo(this, new File(getPath(name)));
		}
		File file = new File(getPath(name));
		try {
			AbstractExternalizable.serializeTo(this, file);
		} catch (IOException e) {
			e.printStackTrace();
		}

		GraphRank graphRank = new GraphRank(dim, beita, gama, initValue, tran);
		double[] ranked = graphRank.rank();

		for (int i = 0; i < dim; i++) {
			Instance instance = new Instance();
			instance.str = train.get(i);
			instance.sim = ranked[i];
			instances.add(instance);
		}

		Collections.sort(instances, new InstanceCompair());

		for (int i = 0; i < train.size(); i++)
			train.set(i, instances.get(i).getStr());
		return train;

	}

	public double tokenSimilarity(String str1, String str2) {
		// Double sim = tokenSimHashMap.get(str1 + "##" + str2);
		// if (sim == null) {
		double sim = SentenceSimilarity.listCosineSimilarity(tokenList(1, tokenizerFactory, str1), tokenList(1, tokenizerFactory, str2));
		// tokenSimHashMap.put(str1 + "##" + str2, sim);
		// }
		return sim;
	}

	public double tokenSimilarity(List<String> str1, List<String> str2) {
		/*
		 * Double sim = tokenSimHashMap.get(str1 + "##" + str2); if (sim == null) {
		 */
		double sim = SentenceSimilarity.listCosineSimilarity(str1, str2);
		// tokenSimHashMap.put(str1 + "##" + str2, sim);
		// }
		return sim;
	}

	public List<String> tokenList(int type, TokenizerFactory tokFactory, String string) {
		/*
		 * if (type == 1 && tokenMap.containsKey(string)) return
		 * tokenMap.get(string); if (type == 2 && featMap.containsKey(string))
		 * return featMap.get(string);
		 */
		List<String> list = new ArrayList<String>();
		Tokenizer tokenizer = tokFactory.tokenizer(string.toCharArray(), 0, string.length());
		String next;
		while ((next = tokenizer.nextToken()) != null) {
			list.add(next);
		}
		/*
		 * if (type == 1) tokenMap.put(string, list); if (type == 2)
		 * featMap.put(string, list);
		 */
		return list;
	}

	static IctSplitTokenizerFactory ictSplitTokenizerFactory = new IctSplitTokenizerFactory();
	static FilterFeatureExtractor featureExtractor = new FilterFeatureExtractor();

	public double featureSimilarity(String string1, String string2) {
		/*
		 * Double sim = featSimHashMap.get(string1 + "##" + string2); if (sim ==
		 * null) {
		 */
		double sim = SentenceSimilarity.listCommonNum(getFeatureList(string1), getFeatureList(string2));
		/*
		 * featSimHashMap.put(string1 + "##" + string2, sim); }
		 */
		return sim;
	}

	public double featureSimilarity(List<String> string1, List<String> string2) {
		/*
		 * Double sim = featSimHashMap.get(string1 + "##" + string2); if (sim ==
		 * null) {
		 */
		double sim = SentenceSimilarity.listCommonNum(string1, string2);
		// featSimHashMap.put(string1 + "##" + string2, sim);
		// }
		return sim;
	}

	static IctSpliting ictSpliting = new IctSpliting();

	public List<String> getFeatureList(String string) {
		/*
		 * if (featureMap.containsKey(string)) return featureMap.get(string);
		 */
		// List<String> list = tokenList(2, ictSplitTokenizerFactory, string);
		List<String> list = ictSpliting.splitToWords(string);
		List<String> tags = ictSpliting.getPosList();
		/*
		 * for (int i = 0; i < list.size(); i++) tags.add("O");
		 */
		List<String> features = new ArrayList<String>();
		// boolean[] select = { true, true, true, false, false, false, true, true,
		// true, false, false, false, false, false, false, false };

		ChainCrfFeatures chainCrfFeatures = featureExtractor.extract(list, tags);

		for (int i = 0; i < list.size(); i++) {
			for (Object feature : chainCrfFeatures.nodeFeatures(i).keySet())
				features.add((String) feature);
		}
		// featureMap.put(string, features);
		return features;

	}

	/*
	 * public static TrainCorpusFilter Instance() throws IOException,
	 * ClassNotFoundException {
	 * 
	 * return (TrainCorpusFilter) AbstractExternalizable.readObject(new
	 * File(path)); }
	 */

	String path = "myDateBase"+File.separator+"corpus"+File.separator;

	public String getPath(String name) {
		return path + name + "_" + time + "_Similarity_filter.matrix";
	}

	public String getReadPath(String name) {
		return path + name + "_Similarity_filter.matrix";
	}

	public TokenizerFactory getTokenizerFactory() {
		return tokenizerFactory;
	}

	public void setTokenizerFactory(TokenizerFactory tokenizerFactory) {
		this.tokenizerFactory = tokenizerFactory;
	}

	public MapSymbolTable getTable() {
		return table;
	}

	public void setTable(MapSymbolTable table) {
		this.table = table;
	}

	private Object writeReplace() {
		return new Externalizer<String>(this);
	}

	private static class Externalizer<F> extends AbstractExternalizable {

		private final TrainCorpusFilter mtrainCorpusFilter;

		public Externalizer(TrainCorpusFilter trainCorpusFilter) {
			mtrainCorpusFilter = trainCorpusFilter;
		}

		public Externalizer() {
			this(null);
		}

		@Override
		public Object read(ObjectInput in) throws IOException, ClassNotFoundException {
			TrainCorpusFilter mtrainCorpusFilter = new TrainCorpusFilter();
			/*
			 * mtrainCorpusFilter.featMap = (HashMap<String, List<String>>)
			 * in.readObject(); mtrainCorpusFilter.tokenMap = (HashMap<String,
			 * List<String>>) in.readObject(); mtrainCorpusFilter.featureMap =
			 * (HashMap<String, List<String>>) in.readObject();
			 */

			/*
			 * List<String> featkey = (List<String>) in.readObject(); List<String>
			 * tokkey = (List<String>) in.readObject(); List<Double> featValue =
			 * (List<Double>) in.readObject(); List<Double> tokValue = (List<Double>)
			 * in.readObject(); for (int i = 0; i < featkey.size(); i++)
			 * mtrainCorpusFilter.featSimHashMap.put(featkey.get(i),
			 * featValue.get(i)); for (int i = 0; i < tokkey.size(); i++)
			 * mtrainCorpusFilter.tokenSimHashMap.put(tokkey.get(i), tokValue.get(i));
			 */

			int trainNum = in.readInt();
			int testNum = in.readInt();
			mtrainCorpusFilter.trainNum = trainNum;
			mtrainCorpusFilter.testNum = testNum;
			mtrainCorpusFilter.tranTokMatrix = readArray(trainNum, trainNum, in);
			mtrainCorpusFilter.tranFeatMatrix = readArray(trainNum, trainNum, in);
			mtrainCorpusFilter.simTokMatrix = readArray(trainNum, testNum, in);
			mtrainCorpusFilter.simFeatMatrix = readArray(trainNum, testNum, in);

			return mtrainCorpusFilter;
		}

		@Override
		public void writeExternal(ObjectOutput out) throws IOException {

			/*
			 * out.writeObject(mtrainCorpusFilter.featMap);
			 * out.writeObject(mtrainCorpusFilter.tokenMap);
			 * out.writeObject(mtrainCorpusFilter.featureMap);
			 */
			/*
			 * List<String> featkey = new ArrayList<String>(); List<String> tokkey =
			 * new ArrayList<String>(); List<Double> featValue = new
			 * ArrayList<Double>(); List<Double> tokValue = new ArrayList<Double>();
			 * for (Entry<String, Double> entry :
			 * mtrainCorpusFilter.featSimHashMap.entrySet()) {
			 * featkey.add(entry.getKey()); featValue.add(entry.getValue()); } for
			 * (Entry<String, Double> entry :
			 * mtrainCorpusFilter.tokenSimHashMap.entrySet()) {
			 * tokkey.add(entry.getKey()); tokValue.add(entry.getValue()); }
			 * out.writeObject(featkey); out.writeObject(featValue);
			 * out.writeObject(tokkey); out.writeObject(tokValue);
			 */
			int trainNum = mtrainCorpusFilter.trainNum;
			int testNum = mtrainCorpusFilter.testNum;
			out.writeInt(trainNum);
			out.writeInt(testNum);

			float[][] tranTokMatrix = mtrainCorpusFilter.tranTokMatrix;
			float[][] tranFeatMatrix = mtrainCorpusFilter.tranFeatMatrix;
			float[][] simTokMatrix = mtrainCorpusFilter.simTokMatrix;
			float[][] simFeatMatrix = mtrainCorpusFilter.simFeatMatrix;

			writArray(trainNum, trainNum, tranTokMatrix, out);
			writArray(trainNum, trainNum, tranFeatMatrix, out);
			writArray(trainNum, testNum, simTokMatrix, out);
			writArray(trainNum, testNum, simFeatMatrix, out);

			/*
			 * out.writeObject(mtrainCorpusFilter.tranList);
			 * out.writeObject(mtrainCorpusFilter.testList);
			 */
		}
	}

	public static void writArray(int row, int col, float[][] aa, ObjectOutput out) throws IOException {
		for (int i = 0; i < row; i++)
			for (int j = 0; j < col; j++)
				out.writeFloat(aa[i][j]);
	}

	public static float[][] readArray(int row, int col, ObjectInput in) throws IOException {
		float[][] aa = new float[row][col];
		for (int i = 0; i < row; i++)
			for (int j = 0; j < col; j++)
				aa[i][j] = in.readFloat();
		return aa;
	}

	public static void writArray(int col, float[] aa, ObjectOutput out) throws IOException {
		for (int j = 0; j < col; j++)
			out.writeFloat(aa[j]);
	}

	public static float[] readArray(int col, ObjectInput in) throws IOException {
		float[] aa = new float[col];
		for (int j = 0; j < col; j++)
			aa[j] = in.readFloat();
		return aa;
	}

	public static class FilterFeatureExtractor implements ChainCrfFeatureExtractor<String>, Serializable {
		final boolean[] select = { true, true, true, false, false, false, true, true, true, false, false, false, false, false, false, false };
		final double[] weigth = {};

		public FilterFeatureExtractor() {
		}

		@Override
		public ChainCrfFeatures<String> extract(List<String> tokens, List<String> tags) {
			return new FilterChainCrfFeatures(tokens, tags);
		}

		static class FilterChainCrfFeatures extends ChainCrfFeatures<String> {

			public FilterChainCrfFeatures(List<String> tokens, List<String> tags) {
				super(tokens, tags);
			}

			@Override
			public Map<String, ? extends Number> edgeFeatures(int n, int k) {
				ObjectToDoubleMap<String> feats = new ObjectToDoubleMap<String>();

				return feats;
			}

			@Override
			public Map<String, ? extends Number> nodeFeatures(int n) {
				ObjectToDoubleMap<String> feats = new ObjectToDoubleMap<String>();
				boolean bos = n == 0;
				boolean eos = (n + 1) >= numTokens();
				boolean bos2 = n < 2;
				boolean eos2 = (n + 2) >= numTokens();

				String token = token(n);
				String prevToken = bos ? null : token(n - 1);
				String nextToken = eos ? null : token(n + 1);

				String posTag = tag(n);
				String prevPosTag = bos ? null : tag(n);
				String nextPosTag = eos ? null : tag(n);

				/*
				 * if (bos) feats.set("BOS", 1.0); if (eos) feats.set("EOS", 1.0); if
				 * (!bos && !eos) feats.set("!BOS!EOS", 1.0);
				 */

				feats.set("TOK_" + token, 1.0);
				if (!bos)

					feats.set("TOK_PREV_" + prevToken, 1.0);
				if (!eos)

					feats.set("TOK_NEXT_" + nextToken, 1.0);

				feats.set("POS_" + posTag, 1.0);
				if (!bos)

					feats.set("POS_PREV_" + prevPosTag, 1.0);
				if (!eos)

					feats.set("POS_NEXT_" + nextPosTag, 1.0);

				return feats;
			}

		}
	}

	public void addTableList(List<List<String>> list, SymbolTable table) {
		for (int i = 0; i < list.size(); i++) {
			List<String> llList = list.get(i);
			for (int j = 0; j < llList.size(); j++)
				table.getOrAddSymbol(llList.get(j));
		}
	}

	private void readMatrix(String name) throws IOException, ClassNotFoundException {
		TrainCorpusFilter trainCorpusFilter = (TrainCorpusFilter) AbstractExternalizable.readObject(new File(getReadPath(name)));
		int trainNum = trainCorpusFilter.trainNum;
		int testNum = trainCorpusFilter.testNum;

		tranFeatMatrix = splitArray(trainCorpusFilter.tranFeatMatrix, trainNum, trainNum, split);
		tranTokMatrix = splitArray(trainCorpusFilter.tranTokMatrix, trainNum, trainNum, split);
		simFeatMatrix = splitArray(trainCorpusFilter.simFeatMatrix, trainNum, testNum, split);
		simTokMatrix = splitArray(trainCorpusFilter.simTokMatrix, trainNum, testNum, split);
	}

	private float[][] splitArray(float[][] aa, int row, int col, int split) {
		float[][] bb = new float[row / split + 1][col / split + 1];
		for (int i = 0; i < row; i++) {
			if (i % split == 0)
				for (int j = 0; j < col; j++) {
					if (i % split == 0)
						bb[i / split][j / split] = aa[i][j];
				}
		}
		return bb;
	}

	public void simFiltingByTable(int type, double alph, double beita, double gama) throws Exception {

	//	readMatrix(name);

		tokenizerFactory = CharacterTokenizerFactory.INSTANCE;

		List<List<String>> tokss = new ArrayList<List<String>>();
		for (int i = 0; i < dim; i++)
			tokss.add(tokenList(1, tokenizerFactory, trainList.get(i)));

		List<List<String>> featss = new ArrayList<List<String>>();
		for (int i = 0; i < dim; i++)
			featss.add(getFeatureList(trainList.get(i)));

		List<List<String>> testTokss = new ArrayList<List<String>>();
		for (int i = 0; i < testNum; i++)
			testTokss.add(tokenList(1, tokenizerFactory, testList.get(i)));

		List<List<String>> testFeatss = new ArrayList<List<String>>();
		for (int i = 0; i < testNum; i++)
			testFeatss.add(getFeatureList(testList.get(i)));

		SymbolTable table = new MapSymbolTable();
		addTableList(testFeatss, table);
		addTableList(testTokss, table);
		addTableList(featss, table);
		addTableList(tokss, table);

		List<Instance> instances = new ArrayList<Instance>();
		List<Instance> testInstances = new ArrayList<Instance>();
		for (int i = 0; i < dim; i++) {
			Instance instance = new Instance();
			instance.vector = getFloatVector(tokss.get(i), table);
			instance.featVector = getFloatVector(featss.get(i), table);
			instances.add(instance);
		}
		for (int i = 0; i < testNum; i++) {
			Instance instance = new Instance();
			instance.vector = getFloatVector(testTokss.get(i), table);
			instance.featVector = getFloatVector(testFeatss.get(i), table);
			testInstances.add(instance);
		}

		if (type == -1 || type == 1 || type == 11 || type == 12) {
			if (type != 12) {
				simTokMatrix = new float[dim][testNum];
				System.out.println("\nComputing simTokMatrix");
				for (int i = 0; i < dim; i++) {
					Instance instance = instances.get(i);
					for (int t = 0; t < testNum; t++) {
						Instance testInstance = testInstances.get(t);
						float toksim = (float) instance.vector.cosine(testInstance.vector);
						simTokMatrix[i][t] = toksim;
					}
				}
			}
			/*
			 * if (update) //AbstractExternalizable.serializeTo(this, new
			 * File(getPath(name))); }
			 */
			if (type != 11) {
				simFeatMatrix = new float[dim][testNum];
				System.out.println("\nComputing simFeatMatrix");
				for (int i = 0; i < dim; i++) {
					Instance instance = instances.get(i);
					if (i % 10 == 0)
						System.out.println(i + "   " + trainList.get(i));
					for (int t = 0; t < testNum; t++) {
						Instance testInstance = testInstances.get(t);
						// float fs = (float)
						// featureSimilarity(instance.getFeatVector(),
						// testInstance.getFeatVector());
						double fs = instance.featVector.cosine(testInstance.featVector);
						// featsim /= 1000;
						simFeatMatrix[i][t] = (float) fs;
					}
				}
				/*
				 * if (update) AbstractExternalizable.serializeTo(this, new
				 * File(getPath(name))); }
				 */

			}
		}
		/*
		 * for (int i = 0; i < dim; i++) { // ######## Instance instance =
		 * instances.get(i); System.out.println(i + "   " + train.get(i));
		 * 
		 * for (int t = 0; t < test.size(); t++) { Instance testInstance =
		 * testInstances.get(t); // System.out.println(i + "  " + t+"   "+ s2);
		 * double toksim = instance.vector.cosine(testInstance.vector);
		 * 
		 * double featsim = featureSimilarity(instance.featVector,
		 * testInstance.featVector); simTokMatrix[i][t] = toksim;
		 * simFeatMatrix[i][t] = featsim; double fs =
		 * instance.featVector.cosine(testInstance.featVector); featsim /= 1000;
		 * double sim = alph * toksim + (1 - alph) * featsim; initValue[i] += sim; }
		 * 
		 * // if (i % 500 == 0) // AbstractExternalizable.serializeTo(this, new
		 * File(path)); }
		 */
		if (type == 2 || type == -1 || type == 22 || type == 21) {
			if (type != 22) {
				tranTokMatrix = new float[dim][dim];

				System.out.println("\nComputing tranTokMatrix");
				for (int i = 0; i < dim; i++) {
					Instance instance1 = instances.get(i);
					for (int j = 0; j < i; j++) {
						Instance instance2 = instances.get(j);
						float toksim = (float) instance1.getVector().cosine(instance2.getVector());
						tranTokMatrix[i][j] = toksim;
						tranTokMatrix[j][i] = toksim;
					}
					tranTokMatrix[i][i] = 0;
				}
				/*
				 * if (update) AbstractExternalizable.serializeTo(this, new
				 * File(getPath(name)));
				 */
			}
			if (type != 21) {
				tranFeatMatrix = new float[dim][dim];
				System.out.println("\nComputing tranFeatMatrix");
				for (int i = 0; i < dim; i++) {
					Instance instance1 = instances.get(i);
					if (i % 50 == 0)
						System.out.println(i + "   " + trainList.get(i));
					for (int j = 0; j < i; j++) {
						Instance instance2 = instances.get(j);
						// float featsim = (float)
						// featureSimilarity(instance1.getFeatVector(),
						// instance2.getFeatVector());
						float featsim = (float) instance1.getFeatVector().cosine(instance2.getFeatVector());
						tranFeatMatrix[i][j] = featsim;
						tranFeatMatrix[j][i] = featsim;
					}
					tranFeatMatrix[i][i] = 0;
				}
			}
		}
		if (update)
			AbstractExternalizable.serializeTo(this, new File(getPath(name)));

		// }
		/*
		 * for (int i = 0; i < dim; i++) { List<String> tokTrani = tokss.get(i);
		 * List<String> featTrani = featss.get(i); System.out.println(i + "   " +
		 * train.get(i));
		 * 
		 * for (int j = 0; j < i; j++) { List<String> tokTranj = tokss.get(j);
		 * List<String> featTranj = featss.get(j); double toksim =
		 * tokenSimilarity(tokTrani, tokTranj); double featsim =
		 * featureSimilarity(featTrani, featTranj);
		 * 
		 * tranFeatMatrix[i][j] = featsim; tranTokMatrix[i][j] = toksim; double sim
		 * = alph * toksim + (1 - alph) * featsim; tran[i][j] = sim; tran[j][i] =
		 * sim;
		 * 
		 * tranFeatMatrix[j][i] = featsim; tranTokMatrix[j][i] = toksim; }
		 * tran[i][i] = 0;
		 * 
		 * tranFeatMatrix[i][i] = 0; tranTokMatrix[i][i] = 0; // if (i % 2000 == 0)
		 * // AbstractExternalizable.serializeTo(this, new File(path)); }
		 */

		/*
		 * double[] initValue = new double[dim]; java.util.Arrays.fill(initValue,
		 * 0); for (int i = 0; i < dim; i++) { // ######## List<String> tokTran =
		 * tokss.get(i); List<String> featTran = featss.get(i); System.out.println(i
		 * + "   " + train.get(i)); for (int t = 0; t < test.size(); t++) {
		 * 
		 * List<String> tokTest = testTokss.get(t); List<String> featTest =
		 * testFeatss.get(t); // System.out.println(i + "  " + t+"   "+ s2); double
		 * toksim = tokenSimilarity(tokTran, tokTest); double featsim =
		 * featureSimilarity(featTran, featTest); featsim /= 1000; double sim = alph
		 * * toksim + (1 - alph) * featsim; initValue[i] += sim; } if (i % 500 == 0)
		 * AbstractExternalizable.serializeTo(this, new File(path)); }
		 */

		/*
		 * double[][] tran = new double[dim][dim]; for (int i = 0; i < dim; i++) {
		 * List<String> tokTrani = tokss.get(i); List<String> featTrani =
		 * featss.get(i); System.out.println(i + "   " + train.get(i)); for (int j =
		 * 0; j < i; j++) { List<String> tokTranj = tokss.get(j); List<String>
		 * featTranj = featss.get(j); double toksim = tokenSimilarity(tokTrani,
		 * tokTranj); double featsim = featureSimilarity(featTrani, featTranj);
		 * double sim = alph * toksim + (1 - alph) * featsim; tran[i][j] = sim;
		 * tran[j][i] = sim; } tran[i][i] = 0; if (i % 2000 == 0)
		 * AbstractExternalizable.serializeTo(this, new File(path)); }
		 */
		initValue = new float[dim];
		simInitValue = new float[dim];
		featInitValue = new float[dim];
		// java.util.Arrays.fill(initValue, 0);
		for (int i = 0; i < dim; i++) {
			double sumTok = 0;
			double sumFeat = 0;
			for (int j = 0; j < testNum; j++) {
				sumTok += simTokMatrix[i][j];
				sumFeat += simFeatMatrix[i][j];
			}
			simInitValue[i] = (float) sumTok;
			featInitValue[i] = (float) sumFeat;
			initValue[i] = (float) (sumTok * alph + sumFeat * (1 - alph));
		}

		tran = new float[dim][dim];
		for (int i = 0; i < dim; i++) {
			for (int j = 0; j < dim; j++) {
				tran[i][j] = (float) (tranTokMatrix[i][j] * alph + tranFeatMatrix[i][j] * (1 - alph));
			}
		}
		FilterResultMatrix resultMatrix = new FilterResultMatrix();
		resultMatrix.dim = dim;
		resultMatrix.tran = tran;
		resultMatrix.init = initValue;
		AbstractExternalizable.serializeTo(resultMatrix, new File("G:\\1.txt"));
	}

	private double featureSimilarity(SparseFloatVector featVector, SparseFloatVector featVector2) {
		int[] no1 = featVector.nonZeroDimensions();
		int[] no2 = featVector2.nonZeroDimensions();
		Map<Integer, Double> map = new HashMap<Integer, Double>(1024);
		for (int i = 0; i < no2.length; i++)
			map.put(no2[i], 1.0);
		List<Integer> com = new ArrayList<Integer>();
		for (int i = 0; i < no1.length; i++) {
			if (map.containsKey(no1[i]))
				com.add(no1[i]);
		}
		double num = 0;
		for (Integer in : com)
			num += featVector.value(in) + featVector2.value(in);
		return num;
	}

	public static class FilterResultMatrix implements Serializable {

		/**
		 * @Fields serialVersionUID : TODO(用一句话描述这个变量表示什么)
		 */

		private static final long serialVersionUID = 488495510437801539L;
		int dim;
		float[] init;
		float[][] tran;

		private Object writeReplace() {
			return new Externalizer<String>(this);
		}

		private static class Externalizer<F> extends AbstractExternalizable {

			private final FilterResultMatrix mfilterResultMatrix;

			public Externalizer(FilterResultMatrix filterResultMatrix) {
				mfilterResultMatrix = filterResultMatrix;
			}

			public Externalizer() {
				this(null);
			}

			@Override
			public Object read(ObjectInput in) throws IOException, ClassNotFoundException {
				FilterResultMatrix filterResultMatrix = new FilterResultMatrix();
				int dim = filterResultMatrix.dim = in.readInt();
				filterResultMatrix.tran = readArray(dim, dim, in);
				filterResultMatrix.init = readArray(dim, in);
				return filterResultMatrix;
			}

			@Override
			public void writeExternal(ObjectOutput out) throws IOException {
				out.writeInt(mfilterResultMatrix.dim);
				writArray(mfilterResultMatrix.dim, mfilterResultMatrix.dim, mfilterResultMatrix.tran, out);
				writArray(mfilterResultMatrix.dim, mfilterResultMatrix.init, out);
			}
		}
	}

	public static void main(String[] args) throws Exception {

		/*
		 * TrainCorpusFilter trainCorpusFilter = new TrainCorpusFilter();
		 * 
		 * COAE2009TargetCorpus cTargetCorpus = new COAE2009TargetCorpus();
		 * 
		 * XValidatingObjectCorpus<Chunking> train =
		 * cTargetCorpus.get2009TargetCorpus(7000, 10, true);
		 * 
		 * Task3Corpus testTask3Corpus = new Task3Corpus();
		 * XValidatingObjectCorpus<Chunking> test =
		 * testTask3Corpus.getTask3Corpus(5000, "D", 10, true, false, true); //
		 * trainCorpusFilter
		 * .setTokenizerFactory(CharacterTokenizerFactory.INSTANCE); //
		 * trainCorpusFilter.setTokenizerFactory(new IctSplitTokenizerFactory());
		 * trainCorpusFilter.setTokenizerFactory(new
		 * TokenNGramTokenizerFactory(CharacterTokenizerFactory.INSTANCE, 1, 2));
		 * trainCorpusFilter.getFilterCorpus(1000, train, test);
		 */

		COAE2009TargetCorpus COAE2009TargetCorpus = new COAE2009TargetCorpus();
		XValidatingObjectCorpus<Chunking> trainCorpus = COAE2009TargetCorpus.get2009TargetCorpus(1, 10000, 0, true);
		Task3Corpus testTask3Corpus = new Task3Corpus();
		TrainCorpusFilter trainCorpusFilter = new TrainCorpusFilter();
		XValidatingObjectCorpus<Chunking> test;
		TokenizerFactory trainFactory = CharacterTokenizerFactory.INSTANCE;
		trainCorpusFilter.setTokenizerFactory(trainFactory);
		// test = testTask3Corpus.getTask3Corpus(10000, "D", 10, true, false, true);
		test = testTask3Corpus.getTask3AllCorpus(1, 10, true, false, true);
		int time = 2;
		int split = 1;
		int type = -1;
		int maxNum = 50;
		trainCorpusFilter.FilterCorpus("test_0_1", type, maxNum, 0.5, 0.9, 0.9, trainCorpus, test, time, true, split);
		// trainCorpusFilter.FilterCorpus("test_0_1", 22, 50, 0.5, 0.9, 0.9,
		// trainCorpus, test, 1, true, 2);

	}

	public List<Chunking> getTrainChunkings() {
		return trainChunkings;
	}

	public void setTrainChunkings(List<Chunking> trainChunkings) {
		this.trainChunkings = trainChunkings;
	}

	public float[][] getTran() {
		return tran;
	}

	public void setTran(float[][] tran) {
		this.tran = tran;
	}

	public float[] getInitValue() {
		return initValue;
	}

	public void setInitValue(float[] initValue) {
		this.initValue = initValue;
	}

	public int getDim() {
		return dim;
	}

	public void setDim(int dim) {
		this.dim = dim;
	}

	public float[][] getTranTokMatrix() {
		return tranTokMatrix;
	}

	public void setTranTokMatrix(float[][] tranTokMatrix) {
		this.tranTokMatrix = tranTokMatrix;
	}

	public float[][] getTranFeatMatrix() {
		return tranFeatMatrix;
	}

	public void setTranFeatMatrix(float[][] tranFeatMatrix) {
		this.tranFeatMatrix = tranFeatMatrix;
	}

	public float[] getSimInitValue() {
		return simInitValue;
	}

	public void setSimInitValue(float[] simInitValue) {
		this.simInitValue = simInitValue;
	}

	public float[] getFeatInitValue() {
		return featInitValue;
	}

	public void setFeatInitValue(float[] featInitValue) {
		this.featInitValue = featInitValue;
	}
}
